package tools.nlp;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;

/**
 * Make OpenNLP Tokenizer easier to use
 * @author Lanjun
 * @version 1.0
 */
public class TokenizerSimple {
	private String defaultfile = "data/openNLP/bin/en-token.bin";
	private Tokenizer tokenizer = null;
	
	public TokenizerSimple() {
		load();
	}
	
	public TokenizerSimple(String filename) {
		this.defaultfile = filename;
		load();
	}
	
	private void load() {
		InputStream modelIn = null;
		TokenizerModel model = null;
		try {
			modelIn = new FileInputStream(defaultfile);
			model = new TokenizerModel(modelIn);
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (modelIn != null) {
				try {
					modelIn.close();
				} catch (IOException e) {
				}
			}
		}
		
		tokenizer = new TokenizerME(model);
	}

	public Tokenizer getTokenizer() {
		return tokenizer;
	}
	
	public static void main(String[] args) {
		String text = "She wasn't";
		TokenizerSimple ts = new TokenizerSimple();
		Tokenizer tokenizer = ts.getTokenizer();
		String[] result = tokenizer.tokenize(text);
		for (String word : result) {
			System.out.println(word);
		}
	}
}
